<!doctype html>
<html class="no-js" lang="en" data-content_root="./">
  <head><meta charset="utf-8"/>
    <meta name="viewport" content="width=device-width,initial-scale=1"/>
    <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="next" title="Frontend Developer Guide" href="frontend_development.html" /><link rel="prev" title="Components and Features" href="components.html" />

    <!-- Generated with Sphinx 7.3.7 and Furo 2024.05.06 -->
        <title>LIT Python API - 🔥LIT 1.0 documentation</title>
      <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=a746c00c" />
    <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=387cc868" />
    <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=36a5483c" />
    <link rel="stylesheet" type="text/css" href="_static/furo_custom.css?v=6173ce67" />
    
    


<style>
  body {
    --color-code-background: #f8f8f8;
  --color-code-foreground: black;
  
  }
  @media not print {
    body[data-theme="dark"] {
      --color-code-background: #202020;
  --color-code-foreground: #d0d0d0;
  
    }
    @media (prefers-color-scheme: dark) {
      body:not([data-theme="light"]) {
        --color-code-background: #202020;
  --color-code-foreground: #d0d0d0;
  
      }
    }
  }
</style></head>
  <body>
    
    <script>
      document.body.dataset.theme = localStorage.getItem("theme") || "auto";
    </script>
    

<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
  <symbol id="svg-toc" viewBox="0 0 24 24">
    <title>Contents</title>
    <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
      <path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
    </svg>
  </symbol>
  <symbol id="svg-menu" viewBox="0 0 24 24">
    <title>Menu</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
      <line x1="3" y1="12" x2="21" y2="12"></line>
      <line x1="3" y1="6" x2="21" y2="6"></line>
      <line x1="3" y1="18" x2="21" y2="18"></line>
    </svg>
  </symbol>
  <symbol id="svg-arrow-right" viewBox="0 0 24 24">
    <title>Expand</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
      <polyline points="9 18 15 12 9 6"></polyline>
    </svg>
  </symbol>
  <symbol id="svg-sun" viewBox="0 0 24 24">
    <title>Light mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
      <circle cx="12" cy="12" r="5"></circle>
      <line x1="12" y1="1" x2="12" y2="3"></line>
      <line x1="12" y1="21" x2="12" y2="23"></line>
      <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
      <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
      <line x1="1" y1="12" x2="3" y2="12"></line>
      <line x1="21" y1="12" x2="23" y2="12"></line>
      <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
      <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
    </svg>
  </symbol>
  <symbol id="svg-moon" viewBox="0 0 24 24">
    <title>Dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
      <path stroke="none" d="M0 0h24v24H0z" fill="none" />
      <path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
    </svg>
  </symbol>
  <symbol id="svg-sun-half" viewBox="0 0 24 24">
    <title>Auto light/dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-shadow">
      <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
      <circle cx="12" cy="12" r="9" />
      <path d="M13 12h5" />
      <path d="M13 15h4" />
      <path d="M13 18h1" />
      <path d="M13 9h4" />
      <path d="M13 6h1" />
    </svg>
  </symbol>
  <symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
    <title>Auto light/dark, in light mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
      class="icon-custom-derived-from-feather-sun-and-tabler-moon">
      <path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
      <line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
      <line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
      <line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
      <line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
      <line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
      <line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
      <line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
      <line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
      <circle cx="14.5" cy="9.55" r="3.6"/>
    </svg>
  </symbol>
  <symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
    <title>Auto light/dark, in dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
      class="icon-custom-derived-from-feather-sun-and-tabler-moon">
      <path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
      <line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
      <line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
      <line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
      <line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
      <line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
      <line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
      <line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
      <line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
      <circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
    </svg>
  </symbol>
  <symbol id="svg-pencil" viewBox="0 0 24 24">
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
      <path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
      <path d="M13.5 6.5l4 4" />
      <path d="M20 21l2 -2l-2 -2" />
      <path d="M17 17l-2 2l2 2" /
    </svg>
  </symbol>
  <symbol id="svg-eye" viewBox="0 0 24 24">
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
      <path stroke="none" d="M0 0h24v24H0z" fill="none" />
      <path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
      <path
        d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
      <path d="M20 21l2 -2l-2 -2" />
      <path d="M17 17l-2 2l2 2" />
    </svg>
  </symbol>
</svg>

<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation">
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc">
<label class="overlay sidebar-overlay" for="__navigation">
  <div class="visually-hidden">Hide navigation sidebar</div>
</label>
<label class="overlay toc-overlay" for="__toc">
  <div class="visually-hidden">Hide table of contents sidebar</div>
</label>

<a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>



<div class="page">
  <header class="mobile-header">
    <div class="header-left">
      <label class="nav-overlay-icon" for="__navigation">
        <div class="visually-hidden">Toggle site navigation sidebar</div>
        <i class="icon"><svg><use href="#svg-menu"></use></svg></i>
      </label>
    </div>
    <div class="header-center">
      <a href="index.html"><div class="brand">🔥LIT 1.0 documentation</div></a>
    </div>
    <div class="header-right">
      <div class="theme-toggle-container theme-toggle-header">
        <button class="theme-toggle">
          <div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
          <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
          <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
          <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
          <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
        </button>
      </div>
      <label class="toc-overlay-icon toc-header-icon" for="__toc">
        <div class="visually-hidden">Toggle table of contents sidebar</div>
        <i class="icon"><svg><use href="#svg-toc"></use></svg></i>
      </label>
    </div>
  </header>
  <aside class="sidebar-drawer">
    <div class="sidebar-container">
      
      <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
  
  
  <span class="sidebar-brand-text">🔥LIT 1.0 documentation</span>
  
</a><form class="sidebar-search-container" method="get" action="search.html" role="search">
  <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
  <input type="hidden" name="check_keywords" value="yes">
  <input type="hidden" name="area" value="default">
</form>
<div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
  <ul class="current">
<li class="toctree-l1"><a class="reference external" href="https://pair-code.github.io/lit/">   Main Site</a></li>
<li class="toctree-l1"><a class="reference internal" href="getting_started.html">   Getting Started</a></li>
<li class="toctree-l1"><a class="reference internal" href="demos.html">   Examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="ui_guide.html">   UI Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="components.html">   Components &amp; Features</a></li>
<li class="toctree-l1 current current-page"><a class="current reference internal" href="#">   Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="frontend_development.html">   Frontend Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="docker.html">   Running in Docker</a></li>
<li class="toctree-l1"><a class="reference internal" href="glossary.html">   Glossary</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">   FAQ</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/pair-code/lit">   GitHub</a></li>
</ul>

</div>
</div>

      </div>
      
    </div>
  </aside>
  <div class="main">
    <div class="content">
      <div class="article-container">
        <a href="#" class="back-to-top muted-link">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
            <path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
          </svg>
          <span>Back to top</span>
        </a>
        <div class="content-icon-container">
          <div class="view-this-page">
  <a class="muted-link" href="_sources/api.md.txt" title="View this page">
    <svg><use href="#svg-eye"></use></svg>
    <span class="visually-hidden">View this page</span>
  </a>
</div>
<div class="theme-toggle-container theme-toggle-content">
            <button class="theme-toggle">
              <div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
              <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
              <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
              <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
              <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
            </button>
          </div>
          <label class="toc-overlay-icon toc-content-icon" for="__toc">
            <div class="visually-hidden">Toggle table of contents sidebar</div>
            <i class="icon"><svg><use href="#svg-toc"></use></svg></i>
          </label>
        </div>
        <article role="main" id="furo-main-content">
          <section class="tex2jax_ignore mathjax_ignore" id="lit-python-api">
<h1>LIT Python API<a class="headerlink" href="#lit-python-api" title="Link to this heading">¶</a></h1>
<!--* freshness: { owner: 'lit-dev' reviewed: '2024-06-24' } *-->
<!-- [TOC] placeholder - DO NOT REMOVE -->
<section id="design-overview">
<h2>Design Overview<a class="headerlink" href="#design-overview" title="Link to this heading">¶</a></h2>
<p>LIT is a modular system, comprising a collection of backend components (written
in Python) and frontend modules (written in TypeScript). Most users will develop
against the Python API, which is documented below and allows LIT to be extended
with custom models, datasets, metrics, counterfactual generators, and more. The
LIT server and components are provided as a library which users can use through
their own demo binaries or via Colab.</p>
<p>The components can also be used as regular Python classes without starting a
server; see <a class="reference internal" href="#using-lit-components-outside-of-lit">below</a> for details.</p>
<p><img alt="LIT system overview" src="_images/lit-system-diagram.svg" /></p>
<p>The LIT backend serves models, data, and interpretability components, each of
which is a Python class implementing a minimal API and relying on the
<a class="reference internal" href="#type-system">spec system</a> to detect fields and verify compatibility. The
server is stateless, but implements a caching layer for model predictions - this
simplifies component design and allows interactive use of large models like BERT
or T5.</p>
<p>The frontend is a stateful single-page app, built using
<a class="reference external" href="https://lit.dev/">Lit</a><a class="footnote-reference brackets" href="#id4" id="id1" role="doc-noteref"><span class="fn-bracket">[</span>1<span class="fn-bracket">]</span></a> for modularity and <a class="reference external" href="https://mobx.js.org/">MobX</a> for
state management. It consists of a core UI framework, a set of shared “services”
which manage persistent state, and a set of independent modules which render
visualizations and support user interaction. For more details, see the
<a class="reference internal" href="ui_guide.html"><span class="std std-doc">UI guide</span></a> and the
<a class="reference internal" href="frontend_development.html"><span class="std std-doc">frontend developer guide</span></a>.</p>
</section>
<section id="adding-models-and-data">
<h2>Adding Models and Data<a class="headerlink" href="#adding-models-and-data" title="Link to this heading">¶</a></h2>
<p>To run LIT with your own models and data, you can create a custom <code class="docutils literal notranslate"><span class="pre">demo.py</span></code>
script that passes these to the LIT server. For example:</p>
<div class="highlight-py notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
  <span class="c1"># MulitiNLIData implements the Dataset API</span>
  <span class="n">datasets</span> <span class="o">=</span> <span class="p">{</span>
      <span class="s1">&#39;mnli_matched&#39;</span><span class="p">:</span> <span class="n">MultiNLIData</span><span class="p">(</span><span class="s1">&#39;/path/to/dev_matched.tsv&#39;</span><span class="p">),</span>
      <span class="s1">&#39;mnli_mismatched&#39;</span><span class="p">:</span> <span class="n">MultiNLIData</span><span class="p">(</span><span class="s1">&#39;/path/to/dev_mismatched.tsv&#39;</span><span class="p">),</span>
  <span class="p">}</span>

  <span class="c1"># NLIModel implements the Model API</span>
  <span class="n">models</span> <span class="o">=</span> <span class="p">{</span>
      <span class="s1">&#39;model_foo&#39;</span><span class="p">:</span> <span class="n">NLIModel</span><span class="p">(</span><span class="s1">&#39;/path/to/model/foo/files&#39;</span><span class="p">),</span>
      <span class="s1">&#39;model_bar&#39;</span><span class="p">:</span> <span class="n">NLIModel</span><span class="p">(</span><span class="s1">&#39;/path/to/model/bar/files&#39;</span><span class="p">),</span>
  <span class="p">}</span>

  <span class="n">lit_demo</span> <span class="o">=</span> <span class="n">lit_nlp</span><span class="o">.</span><span class="n">dev_server</span><span class="o">.</span><span class="n">Server</span><span class="p">(</span><span class="n">models</span><span class="p">,</span> <span class="n">datasets</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="mi">4321</span><span class="p">)</span>
  <span class="n">lit_demo</span><span class="o">.</span><span class="n">serve</span><span class="p">()</span>

<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
  <span class="n">main</span><span class="p">()</span>
</pre></div>
</div>
<p>Conceptually, a dataset is just a list of examples and a model is just a
function that takes examples and returns predictions. The <a class="reference internal" href="#datasets"><code class="docutils literal notranslate"><span class="pre">Dataset</span></code></a>
and <a class="reference internal" href="#models"><code class="docutils literal notranslate"><span class="pre">Model</span></code></a> classes implement this, and provide metadata (see the
<a class="reference internal" href="#type-system">type system</a>) to describe themselves to other components.</p>
<p>For pre-built <code class="docutils literal notranslate"><span class="pre">demo.py</span></code> examples, check out
<a class="reference external" href="https://github.com/PAIR-code/lit/tree/main/lit_nlp/examples">https://github.com/PAIR-code/lit/tree/main/lit_nlp/examples</a></p>
<section id="validating-models-and-data">
<h3>Validating Models and Data<a class="headerlink" href="#validating-models-and-data" title="Link to this heading">¶</a></h3>
<p>Datasets and models can optionally be validated by LIT to ensure that dataset
examples match their spec and that model output values match their spec.
This can be very helpful during development of new model and dataset wrappers
to ensure correct behavior in LIT.</p>
<p>At LIT server startup, the <code class="docutils literal notranslate"><span class="pre">validate</span></code> flag can be used to enable validation.
There are three modes:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">--validate=first</span></code> will check the first example in each dataset.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">--validate=sample</span></code> will validate a sample of 5% of each dataset.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">--validate=all</span></code> will run validation on all examples from all datasets.</p></li>
</ul>
<p>Additionally, if using LIT datasets and models outside of the LIT server,
validation can be called directly through the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/lib/validation.py"><code class="docutils literal notranslate"><span class="pre">validation</span></code></a> module.</p>
</section>
</section>
<section id="datasets">
<h2>Datasets<a class="headerlink" href="#datasets" title="Link to this heading">¶</a></h2>
<p>Datasets (<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/dataset.py"><code class="docutils literal notranslate"><span class="pre">Dataset</span></code></a>) are
just a list of examples, with associated type information following LIT’s
<a class="reference internal" href="#type-system">type system</a>.</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">spec()</span></code> should return a flat dict that describes the fields in each example</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">self._examples</span></code> should be a list of flat dicts</p></li>
</ul>
<p>LIT operates on all examples loaded in the datasets you include in your LIT
server, therefore you should take care to use dataset sizes that can fit into
memory on your backend server and can be displayed in the browser.</p>
<p>NOTE: See the <a class="reference internal" href="faq.html"><span class="std std-doc">FAQ</span></a> for more details on dataset size limitations.</p>
<p>Implementations should subclass
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/dataset.py"><code class="docutils literal notranslate"><span class="pre">Dataset</span></code></a>. Usually this
is just a few lines of code - for example, the following is a complete
implementation for the <a class="reference external" href="https://cims.nyu.edu/~sbowman/multinli/">MultiNLI</a>
dataset:</p>
<div class="highlight-py notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MultiNLIData</span><span class="p">(</span><span class="n">Dataset</span><span class="p">):</span>
<span class="w">  </span><span class="sd">&quot;&quot;&quot;Loader for MultiNLI development set.&quot;&quot;&quot;</span>

  <span class="n">NLI_LABELS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;entailment&#39;</span><span class="p">,</span> <span class="s1">&#39;neutral&#39;</span><span class="p">,</span> <span class="s1">&#39;contradiction&#39;</span><span class="p">]</span>

  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
    <span class="c1"># Read the eval set from a .tsv file as distributed with the GLUE benchmark.</span>
    <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">&#39;</span><span class="se">\t</span><span class="s1">&#39;</span><span class="p">)</span>
    <span class="c1"># Store as a list of dicts, conforming to self.spec()</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">_examples</span> <span class="o">=</span> <span class="p">[{</span>
      <span class="s1">&#39;premise&#39;</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="s1">&#39;sentence1&#39;</span><span class="p">],</span>
      <span class="s1">&#39;hypothesis&#39;</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="s1">&#39;sentence2&#39;</span><span class="p">],</span>
      <span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="s1">&#39;gold_label&#39;</span><span class="p">],</span>
      <span class="s1">&#39;genre&#39;</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="s1">&#39;genre&#39;</span><span class="p">],</span>
    <span class="p">}</span> <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">iterrows</span><span class="p">()]</span>

  <span class="k">def</span> <span class="nf">spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">types</span><span class="o">.</span><span class="n">Spec</span><span class="p">:</span>
    <span class="k">return</span> <span class="p">{</span>
      <span class="s1">&#39;premise&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
      <span class="s1">&#39;hypothesis&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
      <span class="s1">&#39;label&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">CategoryLabel</span><span class="p">(</span><span class="n">vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">NLI_LABELS</span><span class="p">),</span>
      <span class="c1"># We can include additional fields, which don&#39;t have to be used by the model.</span>
      <span class="s1">&#39;genre&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">CategoryLabel</span><span class="p">(),</span>
    <span class="p">}</span>
</pre></div>
</div>
<p>In this example, all four fields (premise, hypothesis, label, and genre) have
string values, but the <a class="reference internal" href="#type-system">semantic types</a> tell LIT a bit more about
how to interpret them:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">premise</span></code> and <code class="docutils literal notranslate"><span class="pre">hypothesis</span></code> should be treated as natural-language text
(<code class="docutils literal notranslate"><span class="pre">TextSegment</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">label</span></code> should be treated as a categorical feature (<code class="docutils literal notranslate"><span class="pre">CategoryLabel</span></code>) with a
fixed, known set of possible values (<code class="docutils literal notranslate"><span class="pre">vocab=self.NLI_LABELS</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">genre</span></code> should be treated as a categorical feature, but with an unknown or
open set of values.</p></li>
</ul>
<p>This implementation uses Pandas to read a TSV file, but you can also use
services like <a class="reference external" href="https://www.tensorflow.org/datasets">TensorFlow Datasets</a> -
simply wrap them in your <code class="docutils literal notranslate"><span class="pre">__init__()</span></code> function.</p>
<p>Note that you can freely add additional features - such as <code class="docutils literal notranslate"><span class="pre">genre</span></code> in the
example above - which the model may not be aware of. The LIT UI will recognize
these features for slicing, binning, etc., and they will also be available to
interpretation components such as custom metrics.</p>
<section id="transformations">
<h3>Transformations<a class="headerlink" href="#transformations" title="Link to this heading">¶</a></h3>
<p>The <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> class also supports a limited set of transformations, similar to
TensorFlow’s
<a class="reference external" href="https://www.tensorflow.org/api_docs/python/tf/data/Dataset">tf.data.Dataset</a>
but more limited in scope and aimed at supporting quick iteration:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">Dataset.slice[start:step:end]</span></code> will return a new <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> with the same
spec and a slice of the datapoints.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">Dataset.sample(n,</span> <span class="pre">seed=42)</span></code> will return a new <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> with the same spec
and a random sample of the datapoints.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">Dataset.remap(field_map:</span> <span class="pre">dict[str,</span> <span class="pre">str])</span></code> will return a new <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> with
renamed fields in both the examples and spec.</p></li>
</ul>
<p>The latter is a shortcut to use datasets matching one model with another; for
example, a dataset with a <code class="docutils literal notranslate"><span class="pre">&quot;document&quot;</span></code> field can be used with a model expecting
a <code class="docutils literal notranslate"><span class="pre">&quot;text&quot;</span></code> input via <code class="docutils literal notranslate"><span class="pre">Dataset.remap({&quot;document&quot;:</span> <span class="pre">&quot;text&quot;})</span></code>.<a class="footnote-reference brackets" href="#why-not-standardize-names" id="id2" role="doc-noteref"><span class="fn-bracket">[</span>2<span class="fn-bracket">]</span></a></p>
</section>
</section>
<section id="models">
<h2>Models<a class="headerlink" href="#models" title="Link to this heading">¶</a></h2>
<p>Models (<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/model.py"><code class="docutils literal notranslate"><span class="pre">Model</span></code></a>) are
functions which take inputs and produce outputs, with associated type
information following LIT’s <a class="reference internal" href="#type-system">type system</a>. The core API consists
of three methods:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">input_spec()</span></code> should return a flat dict that describes necessary input
fields</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">output_spec()</span></code> should return a flat dict that describes the model’s
predictions and any additional outputs</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">predict()</span></code> should take a sequence of inputs (satisfying <code class="docutils literal notranslate"><span class="pre">input_spec()</span></code>) and
yields a parallel sequence of outputs matching <code class="docutils literal notranslate"><span class="pre">output_spec()</span></code>.</p></li>
</ul>
<p>Implementations should subclass
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/model.py"><code class="docutils literal notranslate"><span class="pre">Model</span></code></a>. An example for
<a class="reference external" href="https://cims.nyu.edu/~sbowman/multinli/">MultiNLI</a> might look something like:</p>
<div class="highlight-py notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">NLIModel</span><span class="p">(</span><span class="n">Model</span><span class="p">):</span>
<span class="w">  </span><span class="sd">&quot;&quot;&quot;Wrapper for a Natural Language Inference model.&quot;&quot;&quot;</span>

  <span class="n">NLI_LABELS</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;entailment&#39;</span><span class="p">,</span> <span class="s1">&#39;neutral&#39;</span><span class="p">,</span> <span class="s1">&#39;contradiction&#39;</span><span class="p">]</span>

  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">**</span><span class="n">kw</span><span class="p">):</span>
    <span class="c1"># Load the model into memory so we&#39;re ready for interactive use.</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">_model</span> <span class="o">=</span> <span class="n">_load_my_model</span><span class="p">(</span><span class="n">model_path</span><span class="p">,</span> <span class="o">**</span><span class="n">kw</span><span class="p">)</span>

  <span class="c1">##</span>
  <span class="c1"># LIT API implementations</span>
  <span class="k">def</span> <span class="nf">predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Input</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Preds</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Predict on a stream of examples.&quot;&quot;&quot;</span>
    <span class="n">examples</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_model</span><span class="o">.</span><span class="n">convert_dict_input</span><span class="p">(</span><span class="n">d</span><span class="p">)</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">inputs</span><span class="p">]</span>  <span class="c1"># any custom preprocessing</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model</span><span class="o">.</span><span class="n">predict_examples</span><span class="p">(</span><span class="n">examples</span><span class="p">)</span>  <span class="c1"># returns a dict for each input</span>

  <span class="k">def</span> <span class="nf">input_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">types</span><span class="o">.</span><span class="n">Spec</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Describe the inputs to the model.&quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="p">{</span>
        <span class="s1">&#39;premise&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
        <span class="s1">&#39;hypothesis&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
    <span class="p">}</span>

  <span class="k">def</span> <span class="nf">output_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">types</span><span class="o">.</span><span class="n">Spec</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Describe the model outputs.&quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="p">{</span>
      <span class="c1"># The &#39;parent&#39; keyword tells LIT where to look for gold labels when computing metrics.</span>
      <span class="s1">&#39;probas&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">MulticlassPreds</span><span class="p">(</span><span class="n">vocab</span><span class="o">=</span><span class="n">NLI_LABELS</span><span class="p">,</span> <span class="n">parent</span><span class="o">=</span><span class="s1">&#39;label&#39;</span><span class="p">),</span>
    <span class="p">}</span>
</pre></div>
</div>
<p>Unlike the dataset example, this model implementation is incomplete - you’ll
need to customize <code class="docutils literal notranslate"><span class="pre">predict()</span></code> accordingly with any pre- or post-processing
needed, such as tokenization.</p>
<p>Many deep learning models support a batched prediction behavior. Thus, we
provide the <code class="docutils literal notranslate"><span class="pre">BatchedModel</span></code> class that implements simple batching. Users of this
class must implement the <code class="docutils literal notranslate"><span class="pre">predict_minibatch()</span></code> function, which should convert
a <code class="docutils literal notranslate"><span class="pre">Sequence</span></code> of <code class="docutils literal notranslate"><span class="pre">JsonDict</span></code> objects to the appropriate batch representation
(typically, a <code class="docutils literal notranslate"><span class="pre">Mapping</span></code> of strings to aligned <code class="docutils literal notranslate"><span class="pre">Sequences</span></code> or <code class="docutils literal notranslate"><span class="pre">Tensors</span></code>) before
calling the model. Optionally, you may want to override the
<code class="docutils literal notranslate"><span class="pre">max_minibatch_size()</span></code> function, which determines the batch size.</p>
<p>Note: there are a few additional methods in the model API - see
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/model.py"><code class="docutils literal notranslate"><span class="pre">Model</span></code></a> for details.</p>
<p>If your model is on a remote server, consider using the <code class="docutils literal notranslate"><span class="pre">BatchedRemoteModel</span></code>
base class, which implements parallel batched requests using a thread pool.</p>
<section id="adding-more-outputs">
<h3>Adding more outputs<a class="headerlink" href="#adding-more-outputs" title="Link to this heading">¶</a></h3>
<p>The above example defined a black-box model, with predictions but no access to
internals. If we want a richer view into the model’s behavior, we can add
additional return fields corresponding to hidden-state activations, gradients,
word embeddings, attention, or more. For example, a BERT-based model with
several such features might have the following <code class="docutils literal notranslate"><span class="pre">output_spec()</span></code>:</p>
<div class="highlight-py notranslate"><div class="highlight"><pre><span></span>  <span class="k">def</span> <span class="nf">output_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">types</span><span class="o">.</span><span class="n">Spec</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Describe the model outputs.&quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="p">{</span>
      <span class="c1"># The &#39;parent&#39; keyword tells LIT where to look for gold labels when computing metrics.</span>
      <span class="s1">&#39;probas&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">MulticlassPreds</span><span class="p">(</span><span class="n">vocab</span><span class="o">=</span><span class="n">NLI_LABELS</span><span class="p">,</span> <span class="n">parent</span><span class="o">=</span><span class="s1">&#39;label&#39;</span><span class="p">),</span>
      <span class="c1"># This model returns two different embeddings (activation vectors), but you can easily add more.</span>
      <span class="s1">&#39;output_embs&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">Embeddings</span><span class="p">(),</span>      <span class="c1"># from [CLS] token at top layer</span>
      <span class="s1">&#39;mean_word_embs&#39;</span><span class="p">:</span>  <span class="n">lit_types</span><span class="o">.</span><span class="n">Embeddings</span><span class="p">(),</span>  <span class="c1"># mean of input word embeddings</span>
      <span class="c1"># In LIT, we treat tokens as another model output. There can be more than one,</span>
      <span class="c1"># and the &#39;parent&#39; field describes which input segment they correspond to.</span>
      <span class="s1">&#39;premise_tokens&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">Tokens</span><span class="p">(</span><span class="n">parent</span><span class="o">=</span><span class="s1">&#39;premise&#39;</span><span class="p">),</span>
      <span class="s1">&#39;hypothesis_tokens&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">Tokens</span><span class="p">(</span><span class="n">parent</span><span class="o">=</span><span class="s1">&#39;hypothesis&#39;</span><span class="p">),</span>
      <span class="c1"># Gradients are also returned by the model; &#39;align&#39; here references a Tokens field.</span>
      <span class="s1">&#39;premise_grad&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TokenGradients</span><span class="p">(</span><span class="n">align</span><span class="o">=</span><span class="s1">&#39;premise_tokens&#39;</span><span class="p">),</span>
      <span class="s1">&#39;hypothesis_grad&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TokenGradients</span><span class="p">(</span><span class="n">align</span><span class="o">=</span><span class="s1">&#39;hypothesis_tokens&#39;</span><span class="p">),</span>
      <span class="c1"># Similarly, attention references a token field, but here we want the model&#39;s full &quot;internal&quot;</span>
      <span class="c1"># tokenization, which might be something like: [START] foo bar baz [SEP] spam eggs [END]</span>
      <span class="s1">&#39;tokens&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">Tokens</span><span class="p">(),</span>
      <span class="s1">&#39;attention_layer0&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">AttentionHeads</span><span class="p">(</span><span class="n">align</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;tokens&#39;</span><span class="p">,</span> <span class="s1">&#39;tokens&#39;</span><span class="p">]),</span>
      <span class="s1">&#39;attention_layer1&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">AttentionHeads</span><span class="p">(</span><span class="n">align</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;tokens&#39;</span><span class="p">,</span> <span class="s1">&#39;tokens&#39;</span><span class="p">]),</span>
      <span class="s1">&#39;attention_layer2&#39;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">AttentionHeads</span><span class="p">(</span><span class="n">align</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;tokens&#39;</span><span class="p">,</span> <span class="s1">&#39;tokens&#39;</span><span class="p">]),</span>
      <span class="c1"># ...and so on. Since the spec is just a dictionary of dataclasses, you can populate it</span>
      <span class="c1"># in a loop if you have many similar fields.</span>
    <span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">predict()</span></code> function would return, for each example, additional dict entries
corresponding to each of these fields.</p>
<p>Note: Because tokenization is often tightly coupled with the model code, we
treat it as an intermediate state on the same level as embeddings or attention,
and thus return <code class="docutils literal notranslate"><span class="pre">Tokens</span></code> as a field in the model <em>output</em>. This also allows
models to expose different tokenizations for different inputs, such as
<code class="docutils literal notranslate"><span class="pre">premise_tokens</span></code> and <code class="docutils literal notranslate"><span class="pre">hypothesis_tokens</span></code> above.</p>
<p>LIT components and frontend modules will automatically detect these spec fields
and use them to support additional interpretation methods, such as the embedding
projector or gradient-based salience maps.</p>
<p>You can also implement multi-headed models this way: simply add additional
output fields for each prediction (such as another <code class="docutils literal notranslate"><span class="pre">MulticlassPreds</span></code>), and
they’ll be automatically detected.</p>
<p>See the <a class="reference internal" href="#type-system">type system documentation</a> for more details on available
types and their semantics.</p>
</section>
<section id="optional-inputs">
<h3>Optional inputs<a class="headerlink" href="#optional-inputs" title="Link to this heading">¶</a></h3>
<p>By default, LIT treats <code class="docutils literal notranslate"><span class="pre">input_spec</span></code> fields as required. However, this can be set
to false if you wish to define optional model inputs. For example, a model that
can accept pre-tokenized inputs might have the following spec:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span>    <span class="k">def</span> <span class="nf">input_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">types</span><span class="o">.</span><span class="n">Spec</span><span class="p">:</span>
      <span class="k">return</span> <span class="p">{</span>
          <span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
          <span class="s2">&quot;tokens&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">Tokens</span><span class="p">(</span><span class="n">parent</span><span class="o">=</span><span class="s1">&#39;text&#39;</span><span class="p">,</span> <span class="n">required</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
      <span class="p">}</span>
</pre></div>
</div>
<p>And in the model’s <code class="docutils literal notranslate"><span class="pre">predict()</span></code>, you would have logic to use these and bypass the
tokenizer:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span>    <span class="k">def</span> <span class="nf">predict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Input</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Preds</span><span class="p">]:</span>
      <span class="n">input_tokens</span> <span class="o">=</span> <span class="p">[</span><span class="n">ex</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;tokens&#39;</span><span class="p">)</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">tokenize</span><span class="p">(</span><span class="n">ex</span><span class="p">[</span><span class="s1">&#39;text&#39;</span><span class="p">])</span>
                      <span class="k">for</span> <span class="n">ex</span> <span class="ow">in</span> <span class="n">inputs</span><span class="p">]</span>
      <span class="c1"># ...rest of your predict logic...</span>
</pre></div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">required=False</span></code> can also be used for label fields (such as <code class="docutils literal notranslate"><span class="pre">&quot;label&quot;:</span> <span class="pre">lit_types.CategoryLabel(required=False)</span></code>), though these can also be omitted from
the input spec entirely if they are not needed to compute model outputs.</p>
</section>
</section>
<section id="interpretation-components">
<h2>Interpretation Components<a class="headerlink" href="#interpretation-components" title="Link to this heading">¶</a></h2>
<p>Backend interpretation components include metrics, salience maps, visualization
aids like <a class="reference external" href="https://umap-learn.readthedocs.io/en/latest/">UMAP</a>, and
counterfactual generator plug-ins.</p>
<p>Most such components implement the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/components.py"><code class="docutils literal notranslate"><span class="pre">Interpreter</span></code></a> API.
Conceptually, this is any function that takes a set of datapoints and a model,
and produces some output.<a class="footnote-reference brackets" href="#identity-component" id="id3" role="doc-noteref"><span class="fn-bracket">[</span>3<span class="fn-bracket">]</span></a> For example,
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/gradient_maps.py">local gradient-based salience (GradientNorm)</a>
processes the <code class="docutils literal notranslate"><span class="pre">TokenGradients</span></code> and <code class="docutils literal notranslate"><span class="pre">Tokens</span></code> returned by a model and produces a
list of scores for each token. The Integrated Gradients saliency method
additionally requires a <code class="docutils literal notranslate"><span class="pre">TokenEmbeddings</span></code> input and corresponding output, as
well as a label field <code class="docutils literal notranslate"><span class="pre">Target</span></code> to pin the gradient target to the same class as
an input and corresponding output. See the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/examples/glue/models.py">GLUE models class</a>
for an example of these spec requirements.</p>
<p>The core API involves implementing the <code class="docutils literal notranslate"><span class="pre">run()</span></code> method:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span>  <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
          <span class="n">inputs</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">],</span>
          <span class="n">model</span><span class="p">:</span> <span class="n">lit_model</span><span class="o">.</span><span class="n">Model</span><span class="p">,</span>
          <span class="n">dataset</span><span class="p">:</span> <span class="n">lit_dataset</span><span class="o">.</span><span class="n">Dataset</span><span class="p">,</span>
          <span class="n">model_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
          <span class="n">config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
    <span class="c1"># config is any runtime options to this component, such as a threshold for</span>
    <span class="c1"># (binary) classification metrics.</span>
</pre></div>
</div>
<p>Output from an interpreter component is unconstrained; it’s up to the frontend
component requesting it to process the output correctly. In particular, some
components (such as salience maps) may operate on each example independently,
similar to model predictions, while others (such as metrics) may produce
aggregate summaries of the input set.</p>
<p>Interpreters are also responsible for verifying compatibility by reading the
model and dataset specs; these are also used to determine what fields to operate
on. A typical implementation just loops over the relevant specs. For example,
for
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/gradient_maps.py">simple gradient-based salience</a>
we might have:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span>  <span class="k">def</span> <span class="nf">find_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output_spec</span><span class="p">:</span> <span class="n">Spec</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
    <span class="c1"># Find TokenGradients fields</span>
    <span class="n">grad_fields</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">find_spec_keys</span><span class="p">(</span><span class="n">output_spec</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">TokenGradients</span><span class="p">)</span>

    <span class="c1"># Check that these are aligned to Tokens fields</span>
    <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">grad_fields</span><span class="p">:</span>
      <span class="n">tokens_field</span> <span class="o">=</span> <span class="n">output_spec</span><span class="p">[</span><span class="n">f</span><span class="p">]</span><span class="o">.</span><span class="n">align</span>  <span class="c1"># pytype: disable=attribute-error</span>
      <span class="k">assert</span> <span class="n">tokens_field</span> <span class="ow">in</span> <span class="n">output_spec</span>
      <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">output_spec</span><span class="p">[</span><span class="n">tokens_field</span><span class="p">],</span> <span class="n">types</span><span class="o">.</span><span class="n">Tokens</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">grad_fields</span>

  <span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
          <span class="n">inputs</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">],</span>
          <span class="n">model</span><span class="p">:</span> <span class="n">lit_model</span><span class="o">.</span><span class="n">Model</span><span class="p">,</span>
          <span class="n">dataset</span><span class="p">:</span> <span class="n">lit_dataset</span><span class="o">.</span><span class="n">Dataset</span><span class="p">,</span>
          <span class="n">model_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
          <span class="n">config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Run this component, given a model and input(s).&quot;&quot;&quot;</span>
    <span class="c1"># Find gradient fields to interpret</span>
    <span class="n">output_spec</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">output_spec</span><span class="p">()</span>
    <span class="n">grad_fields</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">find_fields</span><span class="p">(</span><span class="n">output_spec</span><span class="p">)</span>
    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;Found fields for gradient attribution: </span><span class="si">%s</span><span class="s1">&#39;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">grad_fields</span><span class="p">))</span>
    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">grad_fields</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>  <span class="c1"># pylint: disable=g-explicit-length-test</span>
      <span class="k">return</span> <span class="kc">None</span>

    <span class="c1"># do rest of the work to create the salience maps for each available field</span>

    <span class="c1"># return a dtypes.TokenSalience for each input, which has a list of</span>
    <span class="c1"># tokens (from the model) and their associated scores.</span>
</pre></div>
</div>
<p>This design adds some code overhead to interpretation components, but the
benefit is flexibility - Python can be used to specify complex dependencies
between fields, and multiple outputs can be easily supported in a loop.</p>
<section id="metrics">
<h3>Metrics<a class="headerlink" href="#metrics" title="Link to this heading">¶</a></h3>
<p>For metrics, the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/metrics.py"><code class="docutils literal notranslate"><span class="pre">SimpleMetrics</span></code></a>
class implements the spec-matching and input-unpacking logic to satisfy the
general <code class="docutils literal notranslate"><span class="pre">Interpreter</span></code> API. A subclass of <code class="docutils literal notranslate"><span class="pre">SimpleMetrics</span></code> should implement an
<code class="docutils literal notranslate"><span class="pre">is_compatible()</span></code> method and a <code class="docutils literal notranslate"><span class="pre">compute()</span></code> method, which is called on compatible
(prediction, label) pairs and returns a dict of named score fields. For example:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">RegressionMetrics</span><span class="p">(</span><span class="n">SimpleMetrics</span><span class="p">):</span>
<span class="w">  </span><span class="sd">&quot;&quot;&quot;Standard regression metrics.&quot;&quot;&quot;</span>

  <span class="k">def</span> <span class="nf">is_compatible</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_spec</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">LitType</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Return true if compatible with this field.&quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field_spec</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">RegressionScore</span><span class="p">)</span>

  <span class="k">def</span> <span class="nf">compute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
              <span class="n">labels</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
              <span class="n">preds</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
              <span class="n">label_spec</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">Scalar</span><span class="p">,</span>
              <span class="n">pred_spec</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">RegressionScore</span><span class="p">,</span>
              <span class="n">config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Compute metric(s) between labels and predictions.&quot;&quot;&quot;</span>
    <span class="k">del</span> <span class="n">config</span>
    <span class="n">mse</span> <span class="o">=</span> <span class="n">sklearn_metrics</span><span class="o">.</span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">labels</span><span class="p">,</span> <span class="n">preds</span><span class="p">)</span>
    <span class="n">pearsonr</span> <span class="o">=</span> <span class="n">scipy_stats</span><span class="o">.</span><span class="n">pearsonr</span><span class="p">(</span><span class="n">labels</span><span class="p">,</span> <span class="n">preds</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
    <span class="n">spearmanr</span> <span class="o">=</span> <span class="n">scipy_stats</span><span class="o">.</span><span class="n">spearmanr</span><span class="p">(</span><span class="n">labels</span><span class="p">,</span> <span class="n">preds</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
    <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;mse&#39;</span><span class="p">:</span> <span class="n">mse</span><span class="p">,</span> <span class="s1">&#39;pearsonr&#39;</span><span class="p">:</span> <span class="n">pearsonr</span><span class="p">,</span> <span class="s1">&#39;spearmanr&#39;</span><span class="p">:</span> <span class="n">spearmanr</span><span class="p">}</span>
</pre></div>
</div>
<p>The implementation of <code class="docutils literal notranslate"><span class="pre">SimpleMetrics.run()</span></code> uses the <code class="docutils literal notranslate"><span class="pre">parent</span></code> key (see
<a class="reference internal" href="#type-system">type system</a>) in fields of the model’s output spec to find the
appropriate input fields to compare against, and calls <code class="docutils literal notranslate"><span class="pre">compute()</span></code> accordingly
on the unpacked values.</p>
</section>
<section id="generators">
<h3>Generators<a class="headerlink" href="#generators" title="Link to this heading">¶</a></h3>
<p>Conceptually, a generator is just an interpreter that returns new input
examples. These may depend on the input only, as for techniques such as back-
translation, or can involve feedback from the model, such as for adversarial
attacks.</p>
<p>The core generator API is:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Generator</span><span class="p">(</span><span class="n">Interpreter</span><span class="p">):</span>
<span class="w">  </span><span class="sd">&quot;&quot;&quot;Base class for LIT generators.&quot;&quot;&quot;</span>

  <span class="k">def</span> <span class="nf">generate_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                   <span class="n">inputs</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">],</span>
                   <span class="n">model</span><span class="p">:</span> <span class="n">lit_model</span><span class="o">.</span><span class="n">Model</span><span class="p">,</span>
                   <span class="n">dataset</span><span class="p">:</span> <span class="n">lit_dataset</span><span class="o">.</span><span class="n">Dataset</span><span class="p">,</span>
                   <span class="n">config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="n">JsonDict</span><span class="p">]]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Run generation on a set of inputs.</span>

<span class="sd">    Args:</span>
<span class="sd">      inputs: sequence of inputs, following model.input_spec()</span>
<span class="sd">      model: optional model to use to generate new examples.</span>
<span class="sd">      dataset: optional dataset which the current examples belong to.</span>
<span class="sd">      config: optional runtime config.</span>

<span class="sd">    Returns:</span>
<span class="sd">      list of list of new generated inputs, following model.input_spec()</span>
<span class="sd">    &quot;&quot;&quot;</span>
</pre></div>
</div>
<p>Where the output is a list of lists: a set of generated examples for each input.
For convenience, there is also a <code class="docutils literal notranslate"><span class="pre">generate()</span></code> method which takes a single
example and returns a single list; we provide the more general <code class="docutils literal notranslate"><span class="pre">generate_all()</span></code>
API to support model-based generators (such as back-translation) which benefit
from batched requests.</p>
<p>As with other interpreter components, a generator can take custom arguments
through <code class="docutils literal notranslate"><span class="pre">config</span></code>, such as the list of substitutions for the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/word_replacer.py">word replacer</a>.</p>
<section id="backtranslator-generator">
<h4>Backtranslator Generator<a class="headerlink" href="#backtranslator-generator" title="Link to this heading">¶</a></h4>
<p>The <a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/backtranslator.py">backtranslator</a>
generator translates text segment inputs into foreign languages and back to the
source language in order to create paraphrases.
It relies on the Google Cloud Translate API to perform those translations.
To use it, you must have a Google Cloud project and set up Cloud Translation
as described at <a class="reference external" href="https://cloud.google.com/translate/docs/setup">https://cloud.google.com/translate/docs/setup</a>.
Then, download  your application credentials file locally and set the
GOOGLE_APPLICATION_CREDENTIALS environment variable to point to that file.
With that environment variable set to the correct path, LIT can make use of the
backtranlator generator if you pass it as a generator in the Server constructor.</p>
</section>
</section>
<section id="configuration-ui">
<h3>Configuration UI<a class="headerlink" href="#configuration-ui" title="Link to this heading">¶</a></h3>
<p>Interpreter components support an optional <code class="docutils literal notranslate"><span class="pre">config</span></code> option to specify run-time
options, such as the number of samples for LIME or the pivot languages for
back-translation. LIT provides a simple DSL to define these options, which will
auto-generate a form on the frontend. The DSL uses the same
<a class="reference internal" href="#type-system">type system</a> as used to define data and model outputs, and the
<code class="docutils literal notranslate"><span class="pre">config</span></code> argument will be passed a dict with the form values.</p>
<p>For example, the following spec:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span>  <span class="k">def</span> <span class="nf">config_spec</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">types</span><span class="o">.</span><span class="n">Spec</span><span class="p">:</span>
    <span class="k">return</span> <span class="p">{</span>
        <span class="s2">&quot;Pivot languages&quot;</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">SparseMultilabel</span><span class="p">(</span>
            <span class="n">vocab</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;ar&#39;</span><span class="p">,</span> <span class="s1">&#39;bg&#39;</span><span class="p">,</span> <span class="s1">&#39;de&#39;</span><span class="p">,</span> <span class="s1">&#39;el&#39;</span><span class="p">,</span> <span class="s1">&#39;en&#39;</span><span class="p">,</span> <span class="s1">&#39;es&#39;</span><span class="p">,</span> <span class="s1">&#39;fr&#39;</span><span class="p">,</span> <span class="s1">&#39;hi&#39;</span><span class="p">,</span> <span class="s1">&#39;ru&#39;</span><span class="p">,</span> <span class="s1">&#39;sw&#39;</span><span class="p">,</span>
                   <span class="s1">&#39;th&#39;</span><span class="p">,</span> <span class="s1">&#39;tr&#39;</span><span class="p">,</span> <span class="s1">&#39;ur&#39;</span><span class="p">,</span> <span class="s1">&#39;vi&#39;</span><span class="p">,</span> <span class="s1">&#39;zh&#39;</span><span class="p">],</span>
            <span class="n">default</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;de&#39;</span><span class="p">,</span> <span class="s1">&#39;fr&#39;</span><span class="p">]),</span>
        <span class="s2">&quot;Source language&quot;</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="s1">&#39;en&#39;</span><span class="p">),</span>
    <span class="p">}</span>
</pre></div>
</div>
<p>will give this form to configure back-translation:</p>
<p><a class="reference internal" href="_images/backtranslation-form-example.png"><img alt="Back-translation Config Form" class="align-center" src="_images/backtranslation-form-example.png" style="width: 400px;" /></a></p>
<p>Currently <code class="docutils literal notranslate"><span class="pre">config_spec()</span></code> is supported only for generators and salience methods,
though any component can support the <code class="docutils literal notranslate"><span class="pre">config</span></code> argument to its <code class="docutils literal notranslate"><span class="pre">run()</span></code> method,
which can be useful if
<a class="reference internal" href="#using-lit-components-outside-of-lit">running outside of the LIT UI</a>.</p>
<p>The following <a class="reference internal" href="#available-types">types</a> are supported (see
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/client/elements/interpreter_controls.ts">interpreter_controls.ts</a>):</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">Scalar</span></code>, which creates a slider for setting a numeric option. You can
specify the <code class="docutils literal notranslate"><span class="pre">min_val</span></code>, <code class="docutils literal notranslate"><span class="pre">max_val</span></code>, <code class="docutils literal notranslate"><span class="pre">default</span></code>, and <code class="docutils literal notranslate"><span class="pre">step</span></code>, values for the
slider through arguments to the <code class="docutils literal notranslate"><span class="pre">Scalar</span></code> constructor.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">Boolean</span></code> (<code class="docutils literal notranslate"><span class="pre">BooleanLitType</span></code> in TypeScript), which creates a checkbox, with
a <code class="docutils literal notranslate"><span class="pre">default</span></code> value to be set in the constructor.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">CategoryLabel</span></code>, which creates a dropdown with options specified in the
<code class="docutils literal notranslate"><span class="pre">vocab</span></code> argument.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">SparseMultilabel</span></code>, which creates a series of checkboxes for each option
specified in the <code class="docutils literal notranslate"><span class="pre">vocab</span></code> argument.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">TextSegment</span></code>, which creates an input text box for string entry, with an
optional default value from the <code class="docutils literal notranslate"><span class="pre">default</span></code> argument.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">Tokens</span></code>, which creates an input text box for entry of multiple,
comma-separated strings which are parsed into a list of strings to be
supplied to the interpreter.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">SingleFieldMatcher</span></code>, which acts like a <code class="docutils literal notranslate"><span class="pre">CategoryLabel</span></code> but where the vocab
is automatically populated by the names of fields from the data or model
spec. For example, <code class="docutils literal notranslate"><span class="pre">SingleFieldMatcher(spec='dataset',</span> <span class="pre">types=['TextSegment'])</span></code> will give a dropdown with the names of all
<code class="docutils literal notranslate"><span class="pre">TextSegment</span></code> fields in the dataset.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">MultiFieldMatcher</span></code> is similar to <code class="docutils literal notranslate"><span class="pre">SingleFieldMatcher</span></code> except it gives a set
of checkboxes to select one or more matching field names. The returned value
in <code class="docutils literal notranslate"><span class="pre">config</span></code> will be a list of string values.</p></li>
</ul>
<p>The field matching controls can be useful for selecting one or more fields to
operate on. For example,to choose which input fields to perturb, or which output
field of a multi-head model to run an adversarial attack (such as HotFlip)
against.</p>
</section>
</section>
<section id="type-system">
<h2>Type System<a class="headerlink" href="#type-system" title="Link to this heading">¶</a></h2>
<p>LIT passes data around (e.g., between the server and the web app) as flat
records with <code class="docutils literal notranslate"><span class="pre">string</span></code> keys. In Python types these are <code class="docutils literal notranslate"><span class="pre">Mapping[str,</span> <span class="pre">...]</span></code> and in
TypeScript types these are <code class="docutils literal notranslate"><span class="pre">{[key:</span> <span class="pre">string]:</span> <span class="pre">unknown}</span></code>. LIT serializes these
records to JSON when communicating between the server and the web app client. It
is because of this serialization that we introduced LIT’s type system; LIT needs
a way to communicate how to process and understand the semantics of the <em>shape</em>
and (allowable) <em>values</em> for the records being passed around in
<a class="reference external" href="https://www.json.org">JSON’s more limited type system</a>.</p>
<!--
  TODO(b/290782213): Update the serialization discussion above to reflect any
  changes to LIT's wrire format for HTTP APIs.
-->
<!--
  TODO(b/258531316): Update Spec type once converted to a readonly type
-->
<p>The <em>shape</em> of a record – its specific keys and the types of their values
– is defined by a <code class="docutils literal notranslate"><span class="pre">Spec</span></code>; a <code class="docutils literal notranslate"><span class="pre">dict[str,</span> <span class="pre">LitType]</span></code>. Each <code class="docutils literal notranslate"><span class="pre">LitType</span></code> class has
a <code class="docutils literal notranslate"><span class="pre">default</span></code> property whose type annotation describes the type of the <em>value</em> for
that field in a JSON record. <code class="docutils literal notranslate"><span class="pre">LitType</span></code>s are implemented using hierarchical
inheritance; the canonical types can be found in <a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/types.py">types.py</a>, with
parallel implementations in <a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/client/lib/lit_types.ts">lit_types.ts</a>.</p>
<section id="conventions">
<h3>Conventions<a class="headerlink" href="#conventions" title="Link to this heading">¶</a></h3>
<p>LIT supports several different “kinds” of <code class="docutils literal notranslate"><span class="pre">Spec</span></code>s (input vs output vs meta,
etc.), and their use in context has specific implications, described
per base class below.</p>
<ul class="simple">
<li><p><a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/dataset.py"><code class="docutils literal notranslate"><span class="pre">lit_nlp.api.dataset.Dataset</span></code></a></p>
<ul>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> describes the shape of every record in the
<code class="docutils literal notranslate"><span class="pre">Sequence</span></code> returned by <code class="docutils literal notranslate"><span class="pre">Dataset.examples()</span></code>.</p></li>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.init_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Optional[Spec]</span></code></strong> describes the user-configurable
arguments for loading a new instance of this <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> class via the web
app’s UI. Returning <code class="docutils literal notranslate"><span class="pre">None</span></code> or an empty <code class="docutils literal notranslate"><span class="pre">Spec</span></code> means that there is nothing
configurable about how this <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> is loaded, and it will not show up
in the dataset loading section of the web app’s Global Settings.</p></li>
</ul>
</li>
<li><p><a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/model.py"><code class="docutils literal notranslate"><span class="pre">lit_nlp.api.model.Model</span></code></a></p>
<ul>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.input_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> describes the shape required of all records
passed into the <code class="docutils literal notranslate"><span class="pre">Model.predict()</span></code> function via the <code class="docutils literal notranslate"><span class="pre">inputs</span></code> argument. LIT
checks for compatibility between a <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> and a <code class="docutils literal notranslate"><span class="pre">Model</span></code> by ensuring
that <code class="docutils literal notranslate"><span class="pre">Model.input_spec()</span></code> is a subset of <code class="docutils literal notranslate"><span class="pre">Dataset.spec()</span></code>.</p></li>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.output_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> describes the shape of all records returned
by the <code class="docutils literal notranslate"><span class="pre">Model.predict()</span></code> function.</p></li>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.init_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Optional[Spec]</span></code></strong> describes the user-configurable
arguments for loading a new instance of this <code class="docutils literal notranslate"><span class="pre">Model</span></code> class via the web
app’s UI. Returning <code class="docutils literal notranslate"><span class="pre">None</span></code> or an empty <code class="docutils literal notranslate"><span class="pre">Spec</span></code> means that there is nothing
configurable about how this <code class="docutils literal notranslate"><span class="pre">model</span></code> is loaded, and it will not show up in
the model loading section of the web app’s Global Settings.</p></li>
</ul>
</li>
<li><p><a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/components.py"><code class="docutils literal notranslate"><span class="pre">lit_nlp.api.components.[Interpreter</span> <span class="pre">|</span> <span class="pre">Generator]</span></code></a></p>
<ul>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.config_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> describes the user-configurable parameters
for running this component. Returning an empty <code class="docutils literal notranslate"><span class="pre">Spec</span></code> means that this
component always processes inputs in the same way.</p></li>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.meta_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> is essentially unconstrained, but ideally
describes the shape of the records returned by this component’s <code class="docutils literal notranslate"><span class="pre">.run()</span></code>
method. Note that this <code class="docutils literal notranslate"><span class="pre">Spec</span></code> has different semantics depending on the
component type. <code class="docutils literal notranslate"><span class="pre">Interpreter.run()</span></code> typically returns an
<code class="docutils literal notranslate"><span class="pre">Iterable[Mapping[str,</span> <span class="pre">...]]</span></code> of records (i.e., the <code class="docutils literal notranslate"><span class="pre">Mapping</span></code>) with this
shape, because each input corresponds to one interpretation. Whereas
<code class="docutils literal notranslate"><span class="pre">Generator.run()</span></code> typically returns an
<code class="docutils literal notranslate"><span class="pre">Iterable[Iterable[Mapping[str,</span> <span class="pre">...]]]</span></code> of records with this shape,
because each input may enable the generation of one or more new examples.</p></li>
</ul>
</li>
<li><p><a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/components.py"><code class="docutils literal notranslate"><span class="pre">lit_nlp.api.components.Metrics</span></code></a></p>
<ul>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.config_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> describes the user-configurable parameters
for running this component. Returning an empty <code class="docutils literal notranslate"><span class="pre">Spec</span></code> means that this
component always processes inputs in the same way.</p></li>
<li><p><strong><code class="docutils literal notranslate"><span class="pre">.meta_spec()</span> <span class="pre">-&gt;</span> <span class="pre">Spec</span></code></strong> is a slight variation on the tradition <code class="docutils literal notranslate"><span class="pre">Spec</span></code>;
it will always be a <code class="docutils literal notranslate"><span class="pre">Mapping[str,</span> <span class="pre">MetricResult]</span></code> describing the single
record returned by the <code class="docutils literal notranslate"><span class="pre">Metrics.run()</span></code> method for each pair of compatible
keys in the <code class="docutils literal notranslate"><span class="pre">Model.output_spec()</span></code> and <code class="docutils literal notranslate"><span class="pre">Dataset.spec()</span></code>. The <code class="docutils literal notranslate"><span class="pre">MetricResult</span></code>
type also describes how to interpret the values in each record, e.g., if
higher, lower, or numbers closer to zero are better.</p></li>
</ul>
</li>
</ul>
<p>Each <code class="docutils literal notranslate"><span class="pre">LitType</span></code> subclass encapsulates its own semantics (see
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/types.py">types.py</a>), but there are a few conventions all subclasses follow:</p>
<ul class="simple">
<li><p>The <strong><code class="docutils literal notranslate"><span class="pre">align=</span></code> attribute</strong> references another field <em>in the same spec</em> and
implies that both fields have index-aligned elements. For
example, <code class="docutils literal notranslate"><span class="pre">Model.output_spec()</span></code> may contain <code class="docutils literal notranslate"><span class="pre">'tokens':</span> <span class="pre">lit_types.Tokens(...)</span></code>
and <code class="docutils literal notranslate"><span class="pre">'pos':</span> <span class="pre">lit_types.SequenceTags(align='tokens')</span></code>, which references the
“tokens” field. This implies that the “pos” field contains a corresponding
value for every item in “tokens” and that you can access them with numeric
indices. Transitively, this means that using <code class="docutils literal notranslate"><span class="pre">zip(...,</span> <span class="pre">strict=True)</span></code> (in
Python 3.10 and above) will act as a pseudo-validator of this expectation.</p></li>
<li><p>The <strong><code class="docutils literal notranslate"><span class="pre">parent=</span></code> attribute</strong> is <em>typically</em> used by <code class="docutils literal notranslate"><span class="pre">LitType</span></code>s in a
<code class="docutils literal notranslate"><span class="pre">Model.output_spec()</span></code>, and must be a field in the <em>input spec</em> (i.e. the
<code class="docutils literal notranslate"><span class="pre">Dataset.spec()</span></code>) against which this field’s value will be compared. For
example, the <code class="docutils literal notranslate"><span class="pre">Model.output_spec()</span></code> may contain
<code class="docutils literal notranslate"><span class="pre">'probas':</span> <span class="pre">lit_types.MulticlassPreds(parent='label',</span> <span class="pre">...)</span></code> and the
<code class="docutils literal notranslate"><span class="pre">Dataset.spec()</span></code> may contain <code class="docutils literal notranslate"><span class="pre">'label':</span> <span class="pre">lit_types.CategoryLabel()</span></code>, which
means that the <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>’s “label” field contains the ground truth values
for that example, and the class prediction in the “probas” field can be
compared to this label, e.g., by multi-class metrics.</p></li>
<li><p>The <strong><code class="docutils literal notranslate"><span class="pre">vocab=</span></code> attribute</strong> is used to represent the allowable values for
that field, such as a set of classes for a <code class="docutils literal notranslate"><span class="pre">MulticlassPreds</span></code> field, or the
set of labels for a <code class="docutils literal notranslate"><span class="pre">CategoryLabel</span></code> field.</p></li>
<li><p>A field that appears in <em>both</em> the model’s input and output specs is assumed
to represent the same value. This pattern is used for model-based input
manipulation. For example, a
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/examples/glue/models.py">language model</a>
might output <code class="docutils literal notranslate"><span class="pre">'tokens':</span> <span class="pre">lit_types.Tokens(...)</span></code>, and accept as (optional)
input <code class="docutils literal notranslate"><span class="pre">'tokens':</span> <span class="pre">lit_types.Tokens(required=False,</span> <span class="pre">...)</span></code>. An interpretability
component could take output from the former, swap one or more tokens (e.g.
with <code class="docutils literal notranslate"><span class="pre">[MASK]</span></code>), and feed them in the corresponding input field to compute
masked fills.</p></li>
</ul>
</section>
<section id="compatibility-checks">
<h3>Compatibility Checks<a class="headerlink" href="#compatibility-checks" title="Link to this heading">¶</a></h3>
<p>LIT’s type system plays a critical role in ensuring reliability of and
interoperability between the <code class="docutils literal notranslate"><span class="pre">Model</span></code>, <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>, <code class="docutils literal notranslate"><span class="pre">Interpreter</span></code>, <code class="docutils literal notranslate"><span class="pre">Generator</span></code>, and
<code class="docutils literal notranslate"><span class="pre">Metrics</span></code> classes:</p>
<ul class="simple">
<li><p>The <strong>Model-Dataset compatibility check</strong> ensures that the
<code class="docutils literal notranslate"><span class="pre">Model.input_spec()</span></code> is a subset of the <code class="docutils literal notranslate"><span class="pre">Dataset.spec()</span></code>. The base
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/model.py"><code class="docutils literal notranslate"><span class="pre">Model</span></code> class</a> provides a robust and universal implementation of
this check in the <code class="docutils literal notranslate"><span class="pre">is_compatible_with_dataset()</span></code> API, but you can override
this method in your <code class="docutils literal notranslate"><span class="pre">Model</span></code> subclass if you so choose.</p></li>
<li><p>All <a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/components.py"><code class="docutils literal notranslate"><span class="pre">lit_nlp.api.components</span></code> classes</a> provide an
<code class="docutils literal notranslate"><span class="pre">is_compatible</span></code> API to check their compatibility against <code class="docutils literal notranslate"><span class="pre">Model</span></code>s and
<code class="docutils literal notranslate"><span class="pre">Dataset</span></code>s, as appropriate. For example, the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/word_replacer.py"><code class="docutils literal notranslate"><span class="pre">WordReplacer</span></code> generator</a> only checks against the <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>
spec because it does not depend on model outputs, whereas the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/curves.py"><code class="docutils literal notranslate"><span class="pre">Curves</span></code> interpreter</a> checks the <code class="docutils literal notranslate"><span class="pre">Model</span></code> and <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>
because it needs labeled predictions, and the
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/components/gradient_maps.py"><code class="docutils literal notranslate"><span class="pre">GradientDotInput</span></code> interpreter</a> only checks against the
<code class="docutils literal notranslate"><span class="pre">Model.output_spec()</span></code> because it needs data that only the model can provide.</p></li>
</ul>
<p>The LIT web app also uses <code class="docutils literal notranslate"><span class="pre">Spec</span></code> based compatibility checks. Each TypeScript
module defines a <a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/client/core/lit_module.ts"><code class="docutils literal notranslate"><span class="pre">shouldDisplayModule</span></code> function</a> that
returns <code class="docutils literal notranslate"><span class="pre">true</span></code> if any active model-dataset pair provides sufficient information
to support the visualization methods encapsulated by that module. If this
function returns <code class="docutils literal notranslate"><span class="pre">false</span></code>, the module is not displayed in the layout. Note that
this can cause jitter (UI modules appearing, disappearing, reordering, resizing,
etc.) when switching between models or datasets with heterogeneous <code class="docutils literal notranslate"><span class="pre">Spec</span></code>s.</p>
<p>When implementing your own LIT components and modules, you can use
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/lib/utils.py"><code class="docutils literal notranslate"><span class="pre">utils.find_spec_keys()</span></code></a> (Python) and
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/client/lib/utils.ts"><code class="docutils literal notranslate"><span class="pre">findSpecKeys()</span></code></a> (TypeScript) to identify fields of interest in a
<code class="docutils literal notranslate"><span class="pre">Spec</span></code>. These methods recognize and respect subclasses. For example,
<code class="docutils literal notranslate"><span class="pre">utils.find_spec_keys(spec,</span> <span class="pre">Scalar)</span></code> will also match any <code class="docutils literal notranslate"><span class="pre">RegressionScore</span></code>
fields, but <code class="docutils literal notranslate"><span class="pre">utils.find_spec_keys(spec,</span> <span class="pre">RegressionScore)</span></code> will not return all
<code class="docutils literal notranslate"><span class="pre">Scalar</span></code> fields in the <code class="docutils literal notranslate"><span class="pre">Spec</span></code>.</p>
<p>Important: Compatibility checks are performed automatically when
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/app.py">building the <code class="docutils literal notranslate"><span class="pre">LitMetadata</span></code></a> for an instance of <code class="docutils literal notranslate"><span class="pre">LitApp</span></code>,
typically by calling <code class="docutils literal notranslate"><span class="pre">dev_server.Serve()</span></code>. <strong>These checks are not performed when
using components in a raw Python context</strong> (e.g., Colab, Jupyter, a REPL), as
<a class="reference internal" href="#using-lit-components-outside-of-lit">described below</a>, and it is encouraged
that you call these explicitly to ensure compatibility and avoid chasing red
herrings.</p>
</section>
<section id="an-in-depth-example">
<h3>An In-Depth Example<a class="headerlink" href="#an-in-depth-example" title="Link to this heading">¶</a></h3>
<p>Consider the following example from the <a class="reference external" href="https://pair-code.github.io/lit/demos/glue.html">MNLI demo</a>. The
<a class="reference external" href="https://cims.nyu.edu/~sbowman/multinli/">MultiNLI</a> dataset might define the following <code class="docutils literal notranslate"><span class="pre">Spec</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># Dataset.spec()</span>
<span class="p">{</span>
  <span class="s2">&quot;premise&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
  <span class="s2">&quot;hypothesis&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
  <span class="s2">&quot;label&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">CategoryLabel</span><span class="p">(</span>
      <span class="n">vocab</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;entailment&quot;</span><span class="p">,</span> <span class="s2">&quot;neutral&quot;</span><span class="p">,</span> <span class="s2">&quot;contradiction&quot;</span><span class="p">]</span>
  <span class="p">),</span>
  <span class="s2">&quot;genre&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">CategoryLabel</span><span class="p">(),</span>
<span class="p">}</span>
</pre></div>
</div>
<p>An example record in this <code class="docutils literal notranslate"><span class="pre">Dataset</span></code> might be:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># dataset.examples[0]</span>
<span class="p">{</span>
  <span class="s2">&quot;premise&quot;</span><span class="p">:</span> <span class="s2">&quot;Buffet and a la carte available.&quot;</span><span class="p">,</span>
  <span class="s2">&quot;hypothesis&quot;</span><span class="p">:</span> <span class="s2">&quot;It has a buffet.&quot;</span>
  <span class="s2">&quot;label&quot;</span><span class="p">:</span> <span class="s2">&quot;entailment&quot;</span><span class="p">,</span>
  <span class="s2">&quot;genre&quot;</span><span class="p">:</span> <span class="s2">&quot;travel&quot;</span><span class="p">,</span>
<span class="p">}</span>
</pre></div>
</div>
<p>A classification model for this task might have the following <code class="docutils literal notranslate"><span class="pre">input_spec()</span></code> and
<code class="docutils literal notranslate"><span class="pre">output_spec()</span></code>. Notice that the input spec is a subset of the <code class="docutils literal notranslate"><span class="pre">Dataset.spec()</span></code>,
thus LIT considers these to be compatible.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># model.input_spec()</span>
<span class="p">{</span>
  <span class="s2">&quot;premise&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
  <span class="s2">&quot;hypothesis&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">TextSegment</span><span class="p">(),</span>
<span class="p">}</span>

<span class="c1"># model.output_spec()</span>
<span class="p">{</span>
  <span class="s2">&quot;probas&quot;</span><span class="p">:</span> <span class="n">lit_types</span><span class="o">.</span><span class="n">MulticlassPreds</span><span class="p">(</span>
      <span class="n">parent</span><span class="o">=</span><span class="s2">&quot;label&quot;</span><span class="p">,</span>
      <span class="n">vocab</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;entailment&quot;</span><span class="p">,</span> <span class="s2">&quot;neutral&quot;</span><span class="p">,</span> <span class="s2">&quot;contradiction&quot;</span><span class="p">]</span>
  <span class="p">),</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Running this model over the input might yield the following prediction.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># model.predict([dataset.examples[0]])[0]</span>
<span class="p">{</span>
  <span class="s2">&quot;probas&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mf">0.967</span><span class="p">,</span> <span class="mf">0.024</span><span class="p">,</span> <span class="mf">0.009</span><span class="p">],</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Passing this input and the prediction to the <code class="docutils literal notranslate"><span class="pre">ClassificationResults</span></code> interpreter
would yield additional human-readable information as follows.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># classification_results.run(</span>
<span class="c1">#     dataset.examples[:1], model, dataset, [prediction]</span>
<span class="c1"># )[0]</span>
<span class="p">{</span>
  <span class="s2">&quot;probas&quot;</span><span class="p">:</span> <span class="p">{</span>
      <span class="s2">&quot;scores&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mf">0.967</span><span class="p">,</span> <span class="mf">0.024</span><span class="p">,</span> <span class="mf">0.009</span><span class="p">],</span>
      <span class="s2">&quot;predicted_class&quot;</span><span class="p">:</span> <span class="s2">&quot;entailment&quot;</span><span class="p">,</span>
      <span class="s2">&quot;correct&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
  <span class="p">},</span>
<span class="p">}</span>
</pre></div>
</div>
<p><em>See the <a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/examples">examples</a> for more.</em></p>
</section>
<section id="available-types">
<h3>Available types<a class="headerlink" href="#available-types" title="Link to this heading">¶</a></h3>
<p>The full set of <code class="docutils literal notranslate"><span class="pre">LitType</span></code>s is defined in
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/types.py">types.py</a>. Numeric types
such as <code class="docutils literal notranslate"><span class="pre">Integer</span></code> and <code class="docutils literal notranslate"><span class="pre">Scalar</span></code> have predefined ranges that can be overridden
using corresponding <code class="docutils literal notranslate"><span class="pre">min_val</span></code> and <code class="docutils literal notranslate"><span class="pre">max_val</span></code> attributes as seen in
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/examples/penguin/data.py">penguin data</a>
<code class="docutils literal notranslate"><span class="pre">INPUT_SPEC</span></code>. The different types available in LIT are summarized in the table
below.</p>
<p>Note: Bracket syntax, such as <code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[num_tokens]</span></code>, refers to the shapes of
NumPy arrays where each element inside the brackets is an integer.</p>
<div class="table-wrapper colwidths-auto docutils container">
<table class="docutils align-default">
<thead>
<tr class="row-odd"><th class="head"><p>Name</p></th>
<th class="head"><p>Description</p></th>
<th class="head"><p>Value Type</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">TextSegment</span></code></p></td>
<td><p>Natural language text, untokenized.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">GeneratedText</span></code></p></td>
<td><p>Untokenized text, generated from a model (such as seq2seq).</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">URL</span></code></p></td>
<td><p>TextSegment, but interpreted as a URL.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">GeneratedURL</span></code></p></td>
<td><p>Generated TextSegment, but interpreted as a URL (i.e., it maye not be real/is inappropriate as a label).</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">SearchQuery</span></code></p></td>
<td><p>TextSegment, but interpreted as a search query.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">String</span></code></p></td>
<td><p>Opaque string data; ignored by components such as perturbation methods that operate on natural language.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">ReferenceTexts</span></code></p></td>
<td><p>Multiple texts, such as a set of references for summarization or MT.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[tuple[str,</span> <span class="pre">float]]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">GeneratedTextCandidates</span></code></p></td>
<td><p>Multiple generation candidates, such as beam search output from a seq2seq model.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[tuple[str,</span> <span class="pre">float]]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">Tokens</span></code></p></td>
<td><p>Tokenized text.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[str]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">TokenTopKPreds</span></code></p></td>
<td><p>Predicted tokens and their scores, as from a language model or seq2seq model.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[list[tuple[str,</span> <span class="pre">float]]]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">Boolean</span></code></p></td>
<td><p>Boolean value.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">bool</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">Scalar</span></code></p></td>
<td><p>Scalar numeric value.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">float</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">Integer</span></code></p></td>
<td><p>Integer, with a default range from -32768 to +32767. value.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">int</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">ImageBytes</span></code></p></td>
<td><p>Image, represented by a base64 encoded string. LIT also provides <code class="docutils literal notranslate"><span class="pre">JPEGBytes</span></code> and <code class="docutils literal notranslate"><span class="pre">PNGBytes</span></code> types for those specific encodings.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">RegressionScore</span></code></p></td>
<td><p>Scalar value, treated as a regression target or prediction.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">float</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">ReferenceScores</span></code></p></td>
<td><p>Scores for one or more reference texts.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[float]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">CategoryLabel</span></code></p></td>
<td><p>Categorical label, from open or fixed vocabulary.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">str</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">MulticlassPreds</span></code></p></td>
<td><p>Multiclass predicted probabilities.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[num_labels]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">SparseMultilabel</span></code></p></td>
<td><p>Multiple non-exclusive labels, such as a set of attributes.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[str]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">SparseMultilabelPreds</span></code></p></td>
<td><p>Sparse multi-label predictions, represented as scored candidates.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[tuple[str,</span> <span class="pre">float]]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">SequenceTags</span></code></p></td>
<td><p>Sequence tags, aligned to tokens.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[str]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">SpanLabels</span></code></p></td>
<td><p>Span labels, aligned to tokens. Each label is (i,j,label).</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[SpanLabel]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">EdgeLabels</span></code></p></td>
<td><p>Edge labels, aligned to tokens. This is a general way to represent many structured prediction tasks, such as coreference or SRL. See <a class="reference external" href="https://arxiv.org/abs/1905.06316">https://arxiv.org/abs/1905.06316</a></p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[EdgeLabel]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">MultiSegmentAnnotations</span></code></p></td>
<td><p>In-line byte-span annotations, which can span multiple text segments.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">list[AnnotationCluster]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">Embeddings</span></code></p></td>
<td><p>Fixed-length embeddings or model activations.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[emb_dim]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">Gradients</span></code></p></td>
<td><p>Gradients with respect to embeddings or model activations.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[emb_dim]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">TokenEmbeddings</span></code></p></td>
<td><p>Per-token embeddings or model activations.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[num_tokens,</span> <span class="pre">emb_dim]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">TokenGradients</span></code></p></td>
<td><p>Gradients with respect to per-token embeddings or model activations.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[num_tokens,</span> <span class="pre">emb_dim]</span></code></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">ImageGradients</span></code></p></td>
<td><p>Gradients with respect to image pixels.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[image_height,</span> <span class="pre">image_width,</span> <span class="pre">color_channels]</span></code></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">AttentionHeads</span></code></p></td>
<td><p>Attention heads, grouped by layer.</p></td>
<td><p><code class="docutils literal notranslate"><span class="pre">&lt;float&gt;[num_heads,</span> <span class="pre">num_tokens,</span> <span class="pre">num_tokens]</span></code></p></td>
</tr>
</tbody>
</table>
</div>
<p>Values can be plain data, NumPy arrays, or custom dataclasses - see
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/dtypes.py">dtypes.py</a> for further
detail.</p>
<p><em>Note: Note that <code class="docutils literal notranslate"><span class="pre">String</span></code>, <code class="docutils literal notranslate"><span class="pre">Boolean</span></code> and <code class="docutils literal notranslate"><span class="pre">URL</span></code> types in Python are represented
as <code class="docutils literal notranslate"><span class="pre">StringLitType</span></code>, <code class="docutils literal notranslate"><span class="pre">BooleanLitType</span></code> and <code class="docutils literal notranslate"><span class="pre">URLLitType</span></code> in TypeScript to avoid
naming collisions with protected TypeScript keywords.</em></p>
</section>
</section>
<section id="server-configuration">
<h2>Server Configuration<a class="headerlink" href="#server-configuration" title="Link to this heading">¶</a></h2>
<p>Some properties of the LIT frontend can be configured from Python as
<strong>arguments to <code class="docutils literal notranslate"><span class="pre">dev_server.Server()</span></code></strong>. These include:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">page_title</span></code>: set a custom page title.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">canonical_url</span></code>: set a “canonical” URL (such as a shortlink) that will be
used as the base when copying links from the LIT UI.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">default_layout</span></code>: set the default UI layout, by name. See <code class="docutils literal notranslate"><span class="pre">layout.ts</span></code> and
the section below for available layouts.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">demo_mode</span></code>: demo / kiosk mode, which disables some functionality (such as
save/load datapoints) which you may not want to expose to untrusted users.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">inline_doc</span></code>: a markdown string that will be rendered in a documentation
module in the main LIT panel.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">onboard_start_doc</span></code>: a markdown string that will be rendered as the first
panel of the LIT onboarding splash-screen.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">onboard_end_doc</span></code>: a markdown string that will be rendered as the last
panel of the LIT onboarding splash-screen.</p></li>
</ul>
<p>For detailed documentation, see
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/server_flags.py">server_flags.py</a>.</p>
<p>Most Python components (such as <code class="docutils literal notranslate"><span class="pre">Model</span></code>, <code class="docutils literal notranslate"><span class="pre">Dataset</span></code>, and <code class="docutils literal notranslate"><span class="pre">Interpreter</span></code>) also have
a <code class="docutils literal notranslate"><span class="pre">description()</span></code> method which can be used to specify a human-readable
description or help text that will appear in the UI.</p>
<section id="customizing-the-layout">
<h3>Customizing the Layout<a class="headerlink" href="#customizing-the-layout" title="Link to this heading">¶</a></h3>
<p>You can specify custom web app layouts from Python via the <code class="docutils literal notranslate"><span class="pre">layouts=</span></code> attribute.
The value should be a <code class="docutils literal notranslate"><span class="pre">Mapping[str,</span> <span class="pre">LitCanonicalLayout]</span></code>, such as:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PENGUIN_LAYOUT</span> <span class="o">=</span> <span class="n">layout</span><span class="o">.</span><span class="n">LitCanonicalLayout</span><span class="p">(</span>
    <span class="n">upper</span><span class="o">=</span><span class="p">{</span>
        <span class="s1">&#39;Main&#39;</span><span class="p">:</span> <span class="p">[</span>
            <span class="n">modules</span><span class="o">.</span><span class="n">DiveModule</span><span class="p">,</span>
            <span class="n">modules</span><span class="o">.</span><span class="n">DataTableModule</span><span class="p">,</span>
            <span class="n">modules</span><span class="o">.</span><span class="n">DatapointEditorModule</span><span class="p">,</span>
        <span class="p">]</span>
    <span class="p">},</span>
    <span class="n">lower</span><span class="o">=</span><span class="n">layout</span><span class="o">.</span><span class="n">STANDARD_LAYOUT</span><span class="o">.</span><span class="n">lower</span><span class="p">,</span>
    <span class="n">description</span><span class="o">=</span><span class="s1">&#39;Custom layout for the Palmer Penguins demo.&#39;</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
<p>You can pass this to the server as:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">lit_demo</span> <span class="o">=</span> <span class="n">dev_server</span><span class="o">.</span><span class="n">Server</span><span class="p">(</span>
    <span class="n">models</span><span class="p">,</span>
    <span class="n">datasets</span><span class="p">,</span>
    <span class="c1"># other args...</span>
    <span class="n">layouts</span><span class="o">=</span><span class="n">layout</span><span class="o">.</span><span class="n">DEFAULT_LAYOUTS</span> <span class="o">|</span> <span class="p">{</span><span class="s1">&#39;penguins&#39;</span><span class="p">:</span> <span class="n">PENGUIN_LAYOUT</span><span class="p">},</span>
    <span class="n">default_layout</span><span class="o">=</span><span class="s1">&#39;penguins&#39;</span><span class="p">,</span>
    <span class="o">**</span><span class="n">server_flags</span><span class="o">.</span><span class="n">get_flags</span><span class="p">())</span>
<span class="k">return</span> <span class="n">lit_demo</span><span class="o">.</span><span class="n">serve</span><span class="p">()</span>
</pre></div>
</div>
<p>You can see the pre-configured layouts provided by LIT, as well as the list of
modules that can be included in your custom layout in
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/api/layout.py"><code class="docutils literal notranslate"><span class="pre">layout.py</span></code></a>. A
<code class="docutils literal notranslate"><span class="pre">LitCanonicalLayout</span></code> can be defined to achieve four different configurations of
the major content areas:</p>
<ul class="simple">
<li><p>Single-panel: Define only the <code class="docutils literal notranslate"><span class="pre">upper=</span></code> parameter.</p></li>
<li><p>Two-panel, upper/lower: Define the <code class="docutils literal notranslate"><span class="pre">upper=</span></code> and <code class="docutils literal notranslate"><span class="pre">lower=</span></code> parameters.</p></li>
<li><p>Two-panel, left/right: Define the <code class="docutils literal notranslate"><span class="pre">left=</span></code> and <code class="docutils literal notranslate"><span class="pre">upper=</span></code> parameters; the
<code class="docutils literal notranslate"><span class="pre">upper=</span></code> section will be shown on the right.</p></li>
<li><p>Three-panel: Define the <code class="docutils literal notranslate"><span class="pre">left=</span></code>, <code class="docutils literal notranslate"><span class="pre">upper=</span></code>, and <code class="docutils literal notranslate"><span class="pre">lower=</span></code> parameters; the
<code class="docutils literal notranslate"><span class="pre">upper=</span></code> and <code class="docutils literal notranslate"><span class="pre">lower=</span></code> sections will be shown on the right.</p></li>
</ul>
<p>To use a specific layout by default for a given LIT instance, pass the key
(e.g., “simple”, “default”, or the name of a custom layout) as a server flag
when initializing LIT (<code class="docutils literal notranslate"><span class="pre">--default_layout=&lt;layout&gt;</span></code>) or by setting the default
value for that flag in you <code class="docutils literal notranslate"><span class="pre">server.py</span></code> file, e.g.,
<code class="docutils literal notranslate"><span class="pre">flags.FLAGS.set_default('default_layout',</span> <span class="pre">'my_layout_name')</span></code>. The layout can
also be set on-the-fly with the <code class="docutils literal notranslate"><span class="pre">layout=</span></code> URL param, which will take precedence.</p>
<p>Note: The pre-configured layouts are added to every <code class="docutils literal notranslate"><span class="pre">LitApp</span></code> instance using
<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict">dictionary comprehension</a>
where the Mapping passed to the <code class="docutils literal notranslate"><span class="pre">LitApp</span></code> constructor overrides the
pre-configured layouts <code class="docutils literal notranslate"><span class="pre">Mapping</span></code>. Thus, you can remove or change these
pre-configured layouts as you like by passing a <code class="docutils literal notranslate"><span class="pre">Mapping</span></code> where the values of
<code class="docutils literal notranslate"><span class="pre">simple</span></code>, <code class="docutils literal notranslate"><span class="pre">default</span></code>, and/or <code class="docutils literal notranslate"><span class="pre">experimental</span></code> is <code class="docutils literal notranslate"><span class="pre">None</span></code> (to remove) or a
<code class="docutils literal notranslate"><span class="pre">LitCanonicalLayout</span></code> instance (to override) as you desire.</p>
</section>
</section>
<section id="accessing-the-lit-ui-in-notebooks">
<h2>Accessing the LIT UI in Notebooks<a class="headerlink" href="#accessing-the-lit-ui-in-notebooks" title="Link to this heading">¶</a></h2>
<p>As an alternative to running a LIT server and connecting to it through a web
browser, LIT can be used directly inside of python notebook environments, such
as <a class="reference external" href="https://colab.research.google.com/">Colab</a> and
<a class="reference external" href="https://jupyter.org/">Jupyter</a>.</p>
<p>After installing LIT through pip, create a <code class="docutils literal notranslate"><span class="pre">lit_nlp.notebook.LitWidget</span></code> object,
passing in a dict of models and a dict of datasets, similar to the
<code class="docutils literal notranslate"><span class="pre">lit_nlp.dev_server.Server</span></code> constructor. You can optionally provide a height
parameter that specifies the height in pixels to render the LIT UI.</p>
<p>Then, in its own output cell, call the <code class="docutils literal notranslate"><span class="pre">render</span></code> method on the widget object to
render the LIT UI. The LIT UI can be rendered in multiple cells if desired. The
LIT UI can also be rendered in its own browser tab, outside of the notebook, by
passing the parameter <code class="docutils literal notranslate"><span class="pre">open_in_new_tab=True</span></code> to the <code class="docutils literal notranslate"><span class="pre">render</span></code> method. The
<code class="docutils literal notranslate"><span class="pre">render</span></code> method can optionally take in a configuration object to specify
certain options to render the LIT UI using, such as the selected layout,
current display tab, dataset, and models. See
<a class="reference external" href="https://github.com/PAIR-code/lit/blob/main/lit_nlp/notebook.py">notebook.py</a> for details.</p>
<p>The widget has a <code class="docutils literal notranslate"><span class="pre">stop</span></code> method which shuts down the widget’s server. This can be
important for freeing up resources if you plan to create multiple LIT widget
instances in a single notebook. Stopping the server doesn’t disable the model
and dataset instances used by the server; they can still be used in the notebook
and take up the resources they require.</p>
<p>Check out an
<a class="reference external" href="https://colab.research.google.com/github/pair-code/lit/blob/main/lit_nlp/examples/notebooks/LIT_sentiment_classifier.ipynb">example notebook</a>.</p>
</section>
<section id="using-lit-components-outside-of-lit">
<h2>Using LIT components outside of LIT<a class="headerlink" href="#using-lit-components-outside-of-lit" title="Link to this heading">¶</a></h2>
<p>All LIT Python components (models, datasets, interpreters, metrics, generators,
etc.) are standalone classes that do not depend on the serving framework. You
can easily use them from Colab, in scripts, or in your libraries. This can
also be handy for development, as you can test new models or components without
needing to reload the server or click the UI.</p>
<p>For example, to view examples in a dataset:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">lit_nlp.examples.glue</span> <span class="kn">import</span> <span class="n">data</span> <span class="k">as</span> <span class="n">glue_data</span>
<span class="n">dataset</span> <span class="o">=</span> <span class="n">glue_data</span><span class="o">.</span><span class="n">SST2Data</span><span class="p">(</span><span class="s1">&#39;validation&#39;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">dataset</span><span class="o">.</span><span class="n">examples</span><span class="p">)</span>  <span class="c1"># list of records {&quot;sentence&quot;: ..., &quot;label&quot;: ...}</span>
</pre></div>
</div>
<p>And to run inference on a few of them:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">lit_nlp.examples.glue</span> <span class="kn">import</span> <span class="n">models</span> <span class="k">as</span> <span class="n">glue_models</span>

<span class="n">model</span> <span class="o">=</span> <span class="n">glue_models</span><span class="o">.</span><span class="n">SST2Model</span><span class="p">(</span><span class="s2">&quot;/path/to/model/files&quot;</span><span class="p">)</span>
<span class="n">preds</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">dataset</span><span class="o">.</span><span class="n">examples</span><span class="p">[:</span><span class="mi">5</span><span class="p">]))</span>
<span class="c1"># will return records {&quot;probas&quot;: ..., &quot;cls_emb&quot;: ..., ...} for each input</span>
</pre></div>
</div>
<p>Or to compute input salience using
<a class="reference external" href="https://homes.cs.washington.edu/~marcotcr/blog/lime/">LIME</a>:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">lit_nlp.components</span> <span class="kn">import</span> <span class="n">lime_explainer</span>

<span class="n">lime</span> <span class="o">=</span> <span class="n">lime_explainer</span><span class="o">.</span><span class="n">LIME</span><span class="p">()</span>
<span class="n">lime</span><span class="o">.</span><span class="n">run</span><span class="p">([</span><span class="n">dataset</span><span class="o">.</span><span class="n">examples</span><span class="p">[</span><span class="mi">0</span><span class="p">]],</span> <span class="n">model</span><span class="p">,</span> <span class="n">dataset</span><span class="p">)</span>
<span class="c1"># will return {&quot;tokens&quot;: ..., &quot;salience&quot;: ...} for each example given</span>
</pre></div>
</div>
<p>For a full working example in Colab, see <a class="reference external" href="https://colab.research.google.com/github/pair-code/lit/blob/dev/lit_nlp/examples/notebooks/LIT_components_example.ipynb">LIT_components_example.ipynb</a>.</p>
<!-- Links -->
<hr class="footnotes docutils" />
<aside class="footnote-list brackets">
<aside class="footnote brackets" id="id4" role="doc-footnote">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id1">1</a><span class="fn-bracket">]</span></span>
<p>Naming is just a happy coincidence; the Learning Interpretability Tool is
not related to the Lit projects.</p>
</aside>
<aside class="footnote brackets" id="why-not-standardize-names" role="doc-footnote">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id2">2</a><span class="fn-bracket">]</span></span>
<p>We could solve this particular case by
standardizing names, but one still needs to be
explicit if there are multiple segments available,
such as <code class="docutils literal notranslate"><span class="pre">&quot;question&quot;</span></code> and <code class="docutils literal notranslate"><span class="pre">&quot;document&quot;</span></code> for a QA
task.</p>
</aside>
<aside class="footnote brackets" id="identity-component" role="doc-footnote">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id3">3</a><span class="fn-bracket">]</span></span>
<p>A trivial one might just run the model and return
predictions, though in practice we have a separate
endpoint for that.</p>
</aside>
</aside>
</section>
</section>

        </article>
      </div>
      <footer>
        
        <div class="related-pages">
          <a class="next-page" href="frontend_development.html">
              <div class="page-info">
                <div class="context">
                  <span>Next</span>
                </div>
                <div class="title">Frontend Developer Guide</div>
              </div>
              <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
            </a>
          <a class="prev-page" href="components.html">
              <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
              <div class="page-info">
                <div class="context">
                  <span>Previous</span>
                </div>
                
                <div class="title">Components and Features</div>
                
              </div>
            </a>
        </div>
        <div class="bottom-of-page">
          <div class="left-details">
            <div class="copyright">
                Copyright &#169; 2023, Google LLC
            </div>
            Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
            
            <a href="https://github.com/pradyunsg/furo">Furo</a>
            
          </div>
          <div class="right-details">
            
          </div>
        </div>
        
      </footer>
    </div>
    <aside class="toc-drawer">
      
      
      <div class="toc-sticky toc-scroll">
        <div class="toc-title-container">
          <span class="toc-title">
            On this page
          </span>
        </div>
        <div class="toc-tree-container">
          <div class="toc-tree">
            <ul>
<li><a class="reference internal" href="#">LIT Python API</a><ul>
<li><a class="reference internal" href="#design-overview">Design Overview</a></li>
<li><a class="reference internal" href="#adding-models-and-data">Adding Models and Data</a><ul>
<li><a class="reference internal" href="#validating-models-and-data">Validating Models and Data</a></li>
</ul>
</li>
<li><a class="reference internal" href="#datasets">Datasets</a><ul>
<li><a class="reference internal" href="#transformations">Transformations</a></li>
</ul>
</li>
<li><a class="reference internal" href="#models">Models</a><ul>
<li><a class="reference internal" href="#adding-more-outputs">Adding more outputs</a></li>
<li><a class="reference internal" href="#optional-inputs">Optional inputs</a></li>
</ul>
</li>
<li><a class="reference internal" href="#interpretation-components">Interpretation Components</a><ul>
<li><a class="reference internal" href="#metrics">Metrics</a></li>
<li><a class="reference internal" href="#generators">Generators</a><ul>
<li><a class="reference internal" href="#backtranslator-generator">Backtranslator Generator</a></li>
</ul>
</li>
<li><a class="reference internal" href="#configuration-ui">Configuration UI</a></li>
</ul>
</li>
<li><a class="reference internal" href="#type-system">Type System</a><ul>
<li><a class="reference internal" href="#conventions">Conventions</a></li>
<li><a class="reference internal" href="#compatibility-checks">Compatibility Checks</a></li>
<li><a class="reference internal" href="#an-in-depth-example">An In-Depth Example</a></li>
<li><a class="reference internal" href="#available-types">Available types</a></li>
</ul>
</li>
<li><a class="reference internal" href="#server-configuration">Server Configuration</a><ul>
<li><a class="reference internal" href="#customizing-the-layout">Customizing the Layout</a></li>
</ul>
</li>
<li><a class="reference internal" href="#accessing-the-lit-ui-in-notebooks">Accessing the LIT UI in Notebooks</a></li>
<li><a class="reference internal" href="#using-lit-components-outside-of-lit">Using LIT components outside of LIT</a></li>
</ul>
</li>
</ul>

          </div>
        </div>
      </div>
      
      
    </aside>
  </div>
</div><script src="_static/documentation_options.js?v=f2a433a1"></script>
    <script src="_static/doctools.js?v=9a2dae69"></script>
    <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
    <script src="_static/scripts/furo.js?v=4e2eecee"></script>
    </body>
</html>